# 抓取金十数据新闻消息示例
import os, time, random, requests, json
import pandas as pd

# 增加UserAgent，防止被反爬虫拦截
# 安装：pip install fake_useragent
from fake_useragent import UserAgent


class Jin10_News:
    # 助手类
    #
    def __init__(self):
        pass

    def get_header(self):
        # 构造useragent，为避免网络访问时延及报错，fake_useragent.json存放到本地
        location = os.path.dirname(__file__) + '/fake_useragent.json'
        ua = UserAgent()
        # ua = UserAgent()
        return ua.random

    def get_jin10(self):
        '''
        获取金十数据新闻数据
        '''
        url = "https://www.jin10.com/flash_newest.js?t=1667728303028"
        headers = {'User-Agent': self.get_header()}
        import re
        del_list = ['<b>', '</b>', '<br/>', '<span class="section-news">', '</span>']
        # 循环执行，直至获取成功
        loop = True
        while loop == True:
            try:
                response = requests.get(url, headers=headers, timeout=3)
                response.encoding = 'utf-8'
                print(response.text)
            except Exception as e:
                t = random.randint(0, 4)
                print("requests获取数据失败(" + str(e) + ")，延迟" + str(t) + "再试...")
                time.sleep(t)  # 如果刚才获取失败则启用随机休眠停歇，模仿人为访问，避免获取失败
                loop = True
            else:
                if response.content:
                    json_str = response.text[13:-1]  # 截取字符串
                    # print(json_str)
                    if len(json_str) > 50:  # 获取行情信息长度如果小于50字符，则失败
                        kdata = json.loads(json_str)
                        content_list = []
                        for i in kdata:
                            content = {}
                            # 设置不希望包含的字符
                            notincluded = '(VIP|href)'
                            # text = i['data']['content']
                            text = i.get('data').get('content')
                            # 删除标签
                            for delstr in del_list:
                                if text is not None:
                                    text = text.replace(delstr, '')
                                else:
                                    text = ""
                            if len(text) > 5 and not re.search(notincluded, text):
                                content['id'] = i['id']
                                content['content'] = text
                                # content['pic'] = i['data']['pic']
                                content['important'] = i['important']
                                content['channel'] = ",".join(map(str, i['channel']))
                                content_list.append(content)
                        if len(content_list) > 0:
                            df_data = pd.DataFrame(content_list)
                            loop == False
                            return {
                                "success": True,
                                "df_data": df_data
                            }

                        else:
                            return {
                                "success": False,
                                "msg": '获取信息结果为None'
                            }
                else:
                    print("获取信息失败")
                    return {
                        "success": False,
                        "msg": '获取信息失败'
                    }

    def get_keynews(self):
        not_contain = ['']  # 筛选不包含字符串
        contain = ['美元|股票|央行|基金|黄金']  # 筛选包含字符串
        ret = self.get_jin10()
        if ret['success']:
            df_data = ret['df_data']
            df_data['content'] = df_data['content']  # .str[:100]
            if len(not_contain[0]) > 0:
                df_data = df_data[~ df_data['content'].str.contains('|'.join(str(v) for v in not_contain))]
            if len(contain[0]) > 0:
                df_data = df_data[df_data['content'].str.contains('|'.join(str(v) for v in contain))]
            if len(df_data) > 0:
                return df_data
        return None


if __name__ == '__main__':
    news = Jin10_News()
    lastcontent = ''
    while 1:
        ret = news.get_keynews()
        if len(ret) > 0:
            content = ret.iloc[0]['content']
            if content != lastcontent:
                # print(content)
                lastcontent = content
                print(ret)
        # 每隔300秒刷新一次
        time.sleep(300)
